{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clive/tools/scikit-learn/sklearn/cross_validation.py:42: DeprecationWarning: This module has been deprecated in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import seldon.pipeline.auto_transforms as auto\n",
    "import seldon.pipeline.util as sutl\n",
    "import seldon.pipeline.cross_validation as cf\n",
    "from sklearn.pipeline import Pipeline\n",
    "import pandas as pd\n",
    "from seldon import xgb\n",
    "import seldon.pipeline.basic_transforms as bt\n",
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "df = pd.read_csv(\"http://archive.ics.uci.edu/ml/machine-learning-databases/credit-screening/crx.data\",\n",
    "                 header=None,names=[\"A1\",\"A2\",\"A3\",\"A4\",\"A5\",\"A6\",\"A7\",\"A8\",\"A9\",\"A10\",\"A11\",\"A12\",\"A13\",\"A14\",\"A15\",\"target\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A1</th>\n",
       "      <th>A2</th>\n",
       "      <th>A3</th>\n",
       "      <th>A4</th>\n",
       "      <th>A5</th>\n",
       "      <th>A6</th>\n",
       "      <th>A7</th>\n",
       "      <th>A8</th>\n",
       "      <th>A9</th>\n",
       "      <th>A10</th>\n",
       "      <th>A11</th>\n",
       "      <th>A12</th>\n",
       "      <th>A13</th>\n",
       "      <th>A14</th>\n",
       "      <th>A15</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b</td>\n",
       "      <td>30.83</td>\n",
       "      <td>0.000</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>1.25</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>1</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>00202</td>\n",
       "      <td>0</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a</td>\n",
       "      <td>58.67</td>\n",
       "      <td>4.460</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>h</td>\n",
       "      <td>3.04</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>6</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>00043</td>\n",
       "      <td>560</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a</td>\n",
       "      <td>24.50</td>\n",
       "      <td>0.500</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>h</td>\n",
       "      <td>1.50</td>\n",
       "      <td>t</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>00280</td>\n",
       "      <td>824</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b</td>\n",
       "      <td>27.83</td>\n",
       "      <td>1.540</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>3.75</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>5</td>\n",
       "      <td>t</td>\n",
       "      <td>g</td>\n",
       "      <td>00100</td>\n",
       "      <td>3</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b</td>\n",
       "      <td>20.17</td>\n",
       "      <td>5.625</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>1.71</td>\n",
       "      <td>t</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "      <td>s</td>\n",
       "      <td>00120</td>\n",
       "      <td>0</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  A1     A2     A3 A4 A5 A6 A7    A8 A9 A10  A11 A12 A13    A14  A15 target\n",
       "0  b  30.83  0.000  u  g  w  v  1.25  t   t    1   f   g  00202    0      +\n",
       "1  a  58.67  4.460  u  g  q  h  3.04  t   t    6   f   g  00043  560      +\n",
       "2  a  24.50  0.500  u  g  q  h  1.50  t   f    0   f   g  00280  824      +\n",
       "3  b  27.83  1.540  u  g  w  v  3.75  t   t    5   t   g  00100    3      +\n",
       "4  b  20.17  5.625  u  g  w  v  1.71  t   f    0   f   s  00120    0      +"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avg cross validation accuracy  0.860869565217\n"
     ]
    }
   ],
   "source": [
    "tTargetId = bt.Feature_id_transform(zero_based=True,input_feature=\"target\",output_feature=\"targetId\")\n",
    "t_auto = auto.Auto_transform(exclude=[\"target\",\"targetId\"])\n",
    "xg = xgb.XGBoostClassifier(target=\"targetId\",excluded=[\"target\"])\n",
    "cv = cf.Seldon_KFold(xg,10)\n",
    "transformers = [(\"tId\",tTargetId),(\"tAuto\",t_auto),(\"xgb\",cv)]\n",
    "p = Pipeline(transformers)\n",
    "p.fit_transform(df)\n",
    "print \"avg cross validation accuracy \",cv.get_score()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing function at point:  {'n_estimators': 911.78093095347504, 'learning_rate': 0.058005156981055211, 'max_depth': 11.087767061828799} | result: 0.833333\n",
      "Initializing function at point:  {'n_estimators': 809.62014847325509, 'learning_rate': 0.059786507579839755, 'max_depth': 18.652239686420895} | result: 0.821739\n",
      "Initializing function at point:  {'n_estimators': 234.61512357766219, 'learning_rate': 0.026478710189946773, 'max_depth': 6.368707132444162} | result: 0.791304\n",
      "Initializing function at point:  {'n_estimators': 990.19559680689167, 'learning_rate': 0.070816139750618817, 'max_depth': 12.430558275918466} | result: 0.826087\n",
      "Initializing function at point:  {'n_estimators': 120.48141815266844, 'learning_rate': 0.062368361965343309, 'max_depth': 8.3352903838553125} | result: 0.781159\n",
      "Iteration:   1 | Last sampled value:    0.776812 | with parameters:  {'n_estimators': 1000.0, 'learning_rate': 0.034517538483140806, 'max_depth': 7.8948236756820274}\n",
      "               | Current maximum:       0.833333 | with parameters:  {'n_estimators': 911.78093095347504, 'learning_rate': 0.058005156981055211, 'max_depth': 11.087767061828799}\n",
      "               | Time taken: 0 minutes and 15.610965 seconds\n",
      "\n",
      "Iteration:   2 | Last sampled value:    0.782609 | with parameters:  {'n_estimators': 812.57595048397127, 'learning_rate': 0.065281104065071385, 'max_depth': 3.0}\n",
      "               | Current maximum:       0.833333 | with parameters:  {'n_estimators': 911.78093095347504, 'learning_rate': 0.058005156981055211, 'max_depth': 11.087767061828799}\n",
      "               | Time taken: 0 minutes and 8.343149 seconds\n",
      "\n",
      "Iteration:   3 | Last sampled value:    0.756522 | with parameters:  {'n_estimators': 97.356089809422144, 'learning_rate': 0.030571810051815865, 'max_depth': 19.266194641382786}\n",
      "               | Current maximum:       0.833333 | with parameters:  {'n_estimators': 911.78093095347504, 'learning_rate': 0.058005156981055211, 'max_depth': 11.087767061828799}\n",
      "               | Time taken: 0 minutes and 4.359313 seconds\n",
      "\n",
      "Iteration:   4 | Last sampled value:    0.739130 | with parameters:  {'n_estimators': 1000.0, 'learning_rate': 0.059143077608004288, 'max_depth': 15.025603024751947}\n",
      "               | Current maximum:       0.833333 | with parameters:  {'n_estimators': 911.78093095347504, 'learning_rate': 0.058005156981055211, 'max_depth': 11.087767061828799}\n",
      "               | Time taken: 0 minutes and 9.74062 seconds\n",
      "\n",
      "Iteration:   5 | Last sampled value:    0.753623 | with parameters:  {'n_estimators': 444.44461639340267, 'learning_rate': 0.10000000000000001, 'max_depth': 20.0}\n",
      "               | Current maximum:       0.833333 | with parameters:  {'n_estimators': 911.78093095347504, 'learning_rate': 0.058005156981055211, 'max_depth': 11.087767061828799}\n",
      "               | Time taken: 0 minutes and 6.685784 seconds\n",
      "\n",
      "Iteration:   6 | Last sampled value:    0.833333 | with parameters:  {'n_estimators': 898.11551114754445, 'learning_rate': 0.0001, 'max_depth': 3.0}\n",
      "               | Current maximum:       0.833333 | with parameters:  {'n_estimators': 911.78093095347504, 'learning_rate': 0.058005156981055211, 'max_depth': 11.087767061828799}\n",
      "               | Time taken: 0 minutes and 10.840565 seconds\n",
      "\n",
      "Iteration:   7 | Last sampled value:    0.749275 | with parameters:  {'n_estimators': 450.24096015896401, 'learning_rate': 0.056713420171400034, 'max_depth': 15.597410911628616}\n",
      "               | Current maximum:       0.833333 | with parameters:  {'n_estimators': 911.78093095347504, 'learning_rate': 0.058005156981055211, 'max_depth': 11.087767061828799}\n",
      "               | Time taken: 0 minutes and 7.778307 seconds\n",
      "\n",
      "Iteration:   8 | Last sampled value:    0.757971 | with parameters:  {'n_estimators': 876.96222411514532, 'learning_rate': 0.10000000000000001, 'max_depth': 16.450594995368096}\n",
      "               | Current maximum:       0.833333 | with parameters:  {'n_estimators': 911.78093095347504, 'learning_rate': 0.058005156981055211, 'max_depth': 11.087767061828799}\n",
      "               | Time taken: 0 minutes and 7.38782 seconds\n",
      "\n",
      "Iteration:   9 | Last sampled value:    0.827536 | with parameters:  {'n_estimators': 9.9999999999999432, 'learning_rate': 0.0001, 'max_depth': 3.0}\n",
      "               | Current maximum:       0.833333 | with parameters:  {'n_estimators': 911.78093095347504, 'learning_rate': 0.058005156981055211, 'max_depth': 11.087767061828799}\n",
      "               | Time taken: 0 minutes and 1.93751 seconds\n",
      "\n",
      "Iteration:  10 | Last sampled value:    0.831884 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.072441362994008301, 'max_depth': 20.0}\n",
      "               | Current maximum:       0.833333 | with parameters:  {'n_estimators': 911.78093095347504, 'learning_rate': 0.058005156981055211, 'max_depth': 11.087767061828799}\n",
      "               | Time taken: 0 minutes and 2.480383 seconds\n",
      "\n",
      "Iteration:  11 | Last sampled value:    0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 2.001273 seconds\n",
      "\n",
      "Iteration:  12 | Last sampled value:    0.850725 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.10000000000000001, 'max_depth': 3.0}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 1.905338 seconds\n",
      "\n",
      "Iteration:  13 | Last sampled value:    0.742029 | with parameters:  {'n_estimators': 397.46125515446755, 'learning_rate': 0.0001, 'max_depth': 20.0}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 11.601091 seconds\n",
      "\n",
      "Iteration:  14 | Last sampled value:    0.839130 | with parameters:  {'n_estimators': 1000.0, 'learning_rate': 0.046147449096975691, 'max_depth': 3.0}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 9.972134 seconds\n",
      "\n",
      "Iteration:  15 | Last sampled value:    0.820290 | with parameters:  {'n_estimators': 66.822342468251151, 'learning_rate': 0.0001, 'max_depth': 10.55278044795609}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 3.223813 seconds\n",
      "\n",
      "Iteration:  16 | Last sampled value:    0.728986 | with parameters:  {'n_estimators': 1000.0, 'learning_rate': 0.097754508376174595, 'max_depth': 3.0}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 10.451882 seconds\n",
      "\n",
      "Iteration:  17 | Last sampled value:    0.776812 | with parameters:  {'n_estimators': 602.28292862050444, 'learning_rate': 0.0001, 'max_depth': 5.1782258338608367}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 9.278974 seconds\n",
      "\n",
      "Iteration:  18 | Last sampled value:    0.771014 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.10000000000000001, 'max_depth': 20.0}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 1.62759 seconds\n",
      "\n",
      "Iteration:  19 | Last sampled value:    0.740580 | with parameters:  {'n_estimators': 749.53610377769803, 'learning_rate': 0.07904806758164519, 'max_depth': 20.0}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 7.91712 seconds\n",
      "\n",
      "Iteration:  20 | Last sampled value:    0.747826 | with parameters:  {'n_estimators': 855.02778780541553, 'learning_rate': 0.0001, 'max_depth': 20.0}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 25.510986 seconds\n",
      "\n",
      "Iteration:  21 | Last sampled value:    0.823188 | with parameters:  {'n_estimators': 688.30095863851602, 'learning_rate': 0.10000000000000001, 'max_depth': 9.2479711782399932}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 7.006607 seconds\n",
      "\n",
      "Iteration:  22 | Last sampled value:    0.776812 | with parameters:  {'n_estimators': 988.62173128520772, 'learning_rate': 0.10000000000000001, 'max_depth': 9.9887084838701306}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 9.060015 seconds\n",
      "\n",
      "Iteration:  23 | Last sampled value:    0.836232 | with parameters:  {'n_estimators': 675.29328358650321, 'learning_rate': 0.016453461800807232, 'max_depth': 19.746484467931388}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 21.261127 seconds\n",
      "\n",
      "Iteration:  24 | Last sampled value:    0.765217 | with parameters:  {'n_estimators': 210.16554947521249, 'learning_rate': 0.090076931366369128, 'max_depth': 3.0}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 3.277976 seconds\n",
      "\n",
      "Iteration:  25 | Last sampled value:    0.805797 | with parameters:  {'n_estimators': 199.86966672717645, 'learning_rate': 0.0001, 'max_depth': 3.0}\n",
      "               | Current maximum:       0.852174 | with parameters:  {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}\n",
      "               | Time taken: 0 minutes and 3.424776 seconds\n",
      "\n",
      "Optimization finished with maximum: 0.852174, at position: {'n_estimators': 10.0, 'learning_rate': 0.041151080254732048, 'max_depth': 3.0}.\n",
      "Time taken: 3 minutes and 52.452671 seconds.\n",
      "Best accuracy is  0.852173913043\n"
     ]
    }
   ],
   "source": [
    "from seldon.pipeline import bayes_optimize as bopt\n",
    "e_opt = bopt.BayesOptimizer(xg,{'learning_rate': (0.0001, 0.1),'n_estimators': (10,1000),'max_depth':(3,20)},\n",
    "                            param_int=['n_estimators','max_depth'],cv_folds=5)\n",
    "transformers = [(\"tId\",tTargetId),(\"tAuto\",t_auto),(\"bopt\",e_opt)]\n",
    "p = Pipeline(transformers)\n",
    "p.fit_transform(df)\n",
    "print \"Best accuracy is \",e_opt.get_best_score()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
